Option Explicit On 

Imports System.Text.RegularExpressions

Public Class HTMLParser

   '''''''''''''''''''''''''''''''''''''
   ' Public Methods
   '''''''''''''''''''''''''''''''''''''

   Shared Sub ExtractText(ByRef HTML As String, ByVal UnwantedTags As String)

      If InStr(HTML, "<!--start:indexcontent-->") > 0 Then
         HTML = Between(HTML, "<!--start:indexcontent-->", "<!--end:indexcontent-->")
      Else
         ' Remove HTML <head> tag and contents
         HTML = RemoveTag(HTML, "head")
      End If

      ' Remove comments
      HTML = RemoveComments(HTML)

      ' Remove unwanted tags and their contents
      HTML = RemoveTags(HTML, UnwantedTags)

      ' Remove inline tags
      HTML = RegExFunctions.Replace(HTML, "</?(a|abbr|acronym|b|basefont|bdo|big|br|cite|code|dfn|em|font|i|img|input|kbd|label|q|s|samp|select|small|span|strike|strong|sub|sup|textarea|tt|u|var)[^>]*>", "")
      ' Replace all other tags with whitespace
      HTML = RegExFunctions.Replace(HTML, "</?[^>]*>", " ")

   End Sub

   Shared Function ExtractTagContents(ByRef HTML As String, ByVal Tag As String) As String

      Return RegExFunctions.GetSubMatch(HTML, "<" & Tag & ">([^<]*)</" & Tag & ">")

   End Function

   Shared Function ExtractMetaContents(ByRef HTML As String, ByVal Name As String) As String

      Return RegExFunctions.GetSubMatch(HTML, "<meta[^>]+name=[""']?" & Name & "[""']?[^>]+content\s*=\s*[""|']?\s*?([^"">]*)\s*[""|']?[^>]*/?>")

   End Function

   Shared Function RemoveTags(ByRef HTML As String, ByVal Tags As String) As String

      Dim arrTags
      arrTags = Split(Tags, ",")
      Dim Tag
      For Each Tag In arrTags
         Tag = Trim(Tag)
         If Tag <> "" Then
            HTML = RemoveTag(HTML, Tag)
         End If
      Next

      Return HTML

   End Function

   Shared Function RemoveTag(ByRef HTML As String, ByVal Tag As String) As String

      Return RegExFunctions.Replace(HTML, "<" & Tag & "[^>]*?>(.|\s)*?</" & Tag & ">", "")

   End Function

   Shared Function RemoveComments(ByRef HTML As String) As String

      Return RegExFunctions.Replace(HTML, "<!--(.|\s)*?-->", " ")

   End Function

   Shared Function Decode(ByVal HTML As String, ByRef HTMLEntities As SortedList) As String

      If HTML = "" Then Return ""

      HTML = DecodeNamedEntities(HTML, HTMLEntities)
      HTML = DecodeNumericEntities(HTML)

      Return HTML

   End Function


   '''''''''''''''''''''''''''''''''''''
   ' Private Methods
   '''''''''''''''''''''''''''''''''''''

   Private Shared Function DecodeNamedEntities(ByVal HTML As String, ByRef HTMLEntities As SortedList) As String
      ' Decode numeric HTML entities ("&amp; -> "&")

      If HTML = "" Then Return ""

      Dim Entities As MatchCollection
      Entities = RegExFunctions.GetMatches(HTML, "(&[a-zA-Z]{1,10};)")

      ' Do we have at least 1 entity
      If Entities.Count > 0 Then
         Dim Entity As Match
         For Each Entity In Entities
            If Entity.Groups.Count >= 1 Then
               HTML = Replace(HTML, Entity.Groups(0).ToString(), HTMLEntities(Entity.Groups(0).ToString()))
            End If
         Next
      End If

      Return HTML

   End Function

   Private Shared Function DecodeNumericEntities(ByVal HTML As String) As String
      ' Decode numeric HTML entities ("&#65; -> "A")

      If HTML = "" Then Return ""

      Dim i
      For i = 1 To 255
         HTML = Replace(HTML, "&#" & i & ";", Chr(i))
      Next

      Return HTML

   End Function

End Class
